#!/usr/bin/perl

use DBI;
use Cwd;
use POSIX;
use strict;

# get our command line arguments
my $replot_uranus=0;
my $backwards_calibration=0;
my $fill_database=0;
for (my $i=0;$i<=$#ARGV;$i++){
    if ($ARGV[$i] eq "--replot-uranus"){
	$replot_uranus=1;
    } elsif ($ARGV[$i] eq "--backwards-calibration"){
	# backwards calibration is when you calibrate Uranus by
	# 1934's flux scale
	$backwards_calibration=1;
    } elsif ($ARGV[$i] eq "--fill-database"){
	# fill the observation database with info about each of the usable
	# scans
	$fill_database=1;
    }
}

# database details
my $dsn="dbi:Pg:dbname=atca;host=mentok;port=5432";
my $user="postgres";
my $password="";

# Setup the database connection
my $dbh=DBI->connect($dsn,$user,$password,{ RaiseError => 0, AutoCommit => 1});

# these hashes map column names in tables to their order
my %observation;
my %calibrators;
($observation{'source'},$observation{'ra'},$observation{'dec'},$observation{'n_if'},
 $observation{'if_freq'},$observation{'if_nchans'},$observation{'if_bw'},$observation{'time'},
 $observation{'arrayname'},$observation{'project_code'},$observation{'rpfits_file'},
 $observation{'cycle_time'},$observation{'offsource'},$observation{'onsource'},
 $observation{'scantype'},$observation{'calcode'},$observation{'stations'},
 $observation{'header_number'},$observation{'num_processed'},$observation{'processed_ids'},
 $observation{'id'},$observation{'az'},$observation{'el'},$observation{'attenmm'},
 $observation{'subrefl'},$observation{'corr_cfg'},$observation{'airtemp'},
 $observation{'chi'},$observation{'evector'},$observation{'jyperk'},
 $observation{'pntra'},$observation{'pntdec'},$observation{'relhumid'},
 $observation{'smonrms'},$observation{'wind'},$observation{'winddir'},
 $observation{'xtsys'},$observation{'ytsys'},$observation{'xyphase'},$observation{'xcaljy'},
 $observation{'ycaljy'},$observation{'xgtp'},$observation{'ygtp'},$observation{'xsdo'},
 $observation{'ysdo'},$observation{'pointcor'},$observation{'point_time'},
 $observation{'point_source'},$observation{'point_az'},$observation{'point_el'})=
    (0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,
     31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49);
($calibrators{'name'},$calibrators{'ra'},$calibrators{'dec'},$calibrators{'type'},
 $calibrators{'low_freq'},$calibrators{'high_freq'},$calibrators{'bandpass_21cm'},
 $calibrators{'bandpass_13cm'},$calibrators{'bandpass_6cm'},$calibrators{'bandpass_3cm'},
 $calibrators{'bandpass_12mm'},$calibrators{'bandpass_7mm'},$calibrators{'bandpass_3mm'})=
    (50,51,52,53,54,55,56,57,58,59,60,61,62);

my $ura_query="SELECT * FROM observation WHERE source='uranus' AND 10000<any(if_freq) AND ".
    "time>'2009-07-01' AND el>30 ORDER BY time";
my $ura=$dbh->prepare($ura_query);
$ura->execute;

my @done_uranus_ids;

my $working_directory="/data/MENTOK_2/ste616/flux_monitoring/staging_area";

while (my @ura_response=$ura->fetchrow){
    # check that we haven't already looked at this source
    my $already_done=0;
    for (my $i=0;$i<=$#done_uranus_ids;$i++){
	if ($done_uranus_ids[$i]==$ura_response[$observation{'id'}]){
	    $already_done=1;
	    last;
	}
    }
    if ($already_done==1){
	next;
    }
    # check that we have some data on it (ie. all antenna are not offsource)
    my $maxonsource=&max_onsource_time($ura_response[$observation{'onsource'}]);
    if ($maxonsource<1){
	# no good data!
	next;
    }
    if ($ura_response[$observation{'source'}] eq "uranus"){
	# we have found an observation of Uranus, now we check for the nearest
	# events
	my @if_freq=&postgresql_array_get($ura_response[$observation{'if_freq'}]);
	print "\n\n++ found Uranus observation at time: ".$ura_response[$observation{'time'}].
	    " with frequencies ".&array_to_text(@if_freq)." (ID = ".
	    $ura_response[$observation{'id'}].")\n";
	my @nearevents=&find_nearest_events($dbh,$ura_response[$observation{'time'}],
					    @if_freq);
	print "++ nearest events: before = ".$nearevents[0]." after = ".$nearevents[1]."\n";
	# find all the Uranus observations that lie in this range
	my @uranus_ids=&find_nearest_uranusobs($dbh,$ura_response[$observation{'time'}],
					       @nearevents,@if_freq);
	for (my $i=0;$i<=$#uranus_ids;$i++){
	    my @id_response=&get_id_info($dbh,$uranus_ids[$i]);
	    my @id_freqs=&postgresql_array_get($id_response[$observation{'if_freq'}]);
	    print "++ found uranus observation ".$id_response[$observation{'source'}].
		", at time: ".$id_response[$observation{'time'}]." with frequencies ".
		&array_to_text(@id_freqs)." (ID = ".$uranus_ids[$i].")";
	    if (&max_onsource_time($id_response[$observation{'onsource'}])<1){
		print " NO GOOD DATA\n";
		splice @uranus_ids,$i,1;
		$i--;
		next;
	    } else {
		print "\n";
	    }
	    push @done_uranus_ids,$uranus_ids[$i];
	}
	# check for phase cals for each uranus observation
	my @phasecal_ids;
	my @phasecal_sources;
	for (my $i=0;$i<=$#uranus_ids;$i++){
	    my @these_phasecal_ids=&find_phasecals($dbh,$uranus_ids[$i],@if_freq);
	    for (my $j=0;$j<=$#these_phasecal_ids;$j++){
		my @id_response=&get_id_info($dbh,$these_phasecal_ids[$j]);
		my @id_freqs=&postgresql_array_get($id_response[$observation{'if_freq'}]);
		print "++ for uranus observation ID ".$uranus_ids[$i].", found phase ".
		    "calibration with source ".$id_response[$observation{'source'}].
		    " at time: ".$id_response[$observation{'time'}]." with frequencies ".
		    &array_to_text(@id_freqs)." (ID = ".$these_phasecal_ids[$j].")";
		if (&max_onsource_time($id_response[$observation{'onsource'}])<1){
		    print " NO GOOD DATA\n";
		    next;
		} else {
		    print "\n";
		}
		push @phasecal_ids,$these_phasecal_ids[$j];
		my $already_found=0;
		for (my $k=0;$k<=$#phasecal_sources;$k++){
		    if ($phasecal_sources[$k] eq $id_response[$observation{'source'}]){
			$already_found=1;
			last;
		    }
		}
		if ($already_found==0){
		    push @phasecal_sources,$id_response[$observation{'source'}];
		}
	    }
	}
	if ($#phasecal_ids<0){
	    print "-- no phase calibration is possible\n";
	}
	# now find a flux calibrator in the event range
	my @fluxcal_ids=&find_nearest_fluxcals($dbh,$ura_response[$observation{'time'}],
					       @nearevents,@if_freq);
	for (my $i=0;$i<=$#fluxcal_ids;$i++){
	    my @id_response=&get_id_info($dbh,$fluxcal_ids[$i]);
	    my @id_freqs=&postgresql_array_get($id_response[$observation{'if_freq'}]);
	    print "++ found flux calibrator ".$id_response[$observation{'source'}].
		", at time: ".$id_response[$observation{'time'}]." with frequencies ".
		&array_to_text(@id_freqs)." (ID = ",$fluxcal_ids[$i].")";
	    if (&max_onsource_time($id_response[$observation{'onsource'}])<1){
		print " NO GOOD DATA\n";
		splice @fluxcal_ids,$i,1;
		$i--;
		next;
	    } else {
		print "\n";
	    }
	}
	if ($#fluxcal_ids<0){
	    print "-- no flux calibrator observations were found\n";
	    next;
	}
	# now find a bandpass calibrator
	my @bandpasscal_ids=&find_nearest_bandpasscals($dbh,$ura_response[$observation{'time'}],
						    @nearevents,@if_freq);
	my $bandpass_source;
	for (my $i=0;$i<=$#bandpasscal_ids;$i++){
	    my @bid_response=&get_id_info($dbh,$bandpasscal_ids[$i]);
	    my @bid_freqs=&postgresql_array_get($bid_response[$observation{'if_freq'}]);
	    print "++ found bandpass calibrator ".$bid_response[$observation{'source'}].
		", at time: ".$bid_response[$observation{'time'}]." with frequencies ".
		&array_to_text(@bid_freqs)." (ID = ",$bandpasscal_ids[$i].")";
	    $bandpass_source=$bid_response[$observation{'source'}];
	    if (&max_onsource_time($bid_response[$observation{'onsource'}])<1){
		print " NO GOOD DATA\n";
		splice @bandpasscal_ids,$i,1;
		$i--;
		next;
	    } else {
		print "\n";
	    }
	}
	if ($#bandpasscal_ids<0){
	    print "-- no bandpass calibrator observations were found\n";
	    next;
	}
	# if we get here, we're ready to do some calibration!
	if ($fill_database==1){
	    # we extract each of the datasets separately and get information
	    # on each of them, and stick that into the database
	    # first, Uranus observations
	    &fill_database($dbh,$working_directory,@uranus_ids);
	    # then the flux calibrators
	    &fill_database($dbh,$working_directory,@fluxcal_ids);
	    # and the bandpass calibrators
	    &fill_database($dbh,$working_directory,@bandpasscal_ids);
	    # and the phase calibrators
	    &fill_database($dbh,$working_directory,@phasecal_ids);
	    # and then we go to the next set of sources since we don't want
	    # to do any data reduction
	    next;
	}
	# extract all the data first
	my $datadir=&extract_all_data($dbh,$working_directory,@uranus_ids,@phasecal_ids,@fluxcal_ids,
				      @bandpasscal_ids);
	print "++ data is extracted to ".$datadir."\n";

	# from now on we'll go to the directory where the data is so we can stop worrying
	# about MIRIAD's inconsistent handling of long paths
	my $olddir=&change_directory($datadir);

	my $thisdatadir=$datadir;
	$datadir=".";

	if ($replot_uranus==0){
	    # now calibrate on the bandpass calibrator
	    print "++ calibrate bandpass calibrator\n";
	    my @bandifs;
	    for (my $i=0;$i<=$#if_freq;$i++){
		$bandifs[$i]=1;
	    }
	    &calibrate($bandpass_source,"p",2,$datadir,$#if_freq,@bandifs,@if_freq);
	    
	    # copy the bandpass solution to all the other sources
	    &copy_bandpass_solution($bandpass_source,"1934-638",$datadir,@if_freq);
	    &copy_bandpass_solution($bandpass_source,"uranus",$datadir,@if_freq);
	    for (my $i=0;$i<=$#phasecal_sources;$i++){
		&copy_bandpass_solution($bandpass_source,$phasecal_sources[$i],$datadir,@if_freq);
	    }	    
	    
	    # calibrate 1934 now
	    for (my $i=0;$i<=$#if_freq;$i++){
		$bandifs[$i]=0;
	    }
	    &calibrate("1934-638","f",2,$datadir,$#if_freq,@bandifs,@if_freq);
	    
	    # and the phase calibrators
	    for (my $i=0;$i<=$#phasecal_sources;$i++){
		&calibrate($phasecal_sources[$i],"p",2,$datadir,$#if_freq,@bandifs,@if_freq);
	    }
	    
	    # we need to keep one amplitude scaling for all the datasets, so we
	    # scale everything according to 1934's gains
	    my @datasets=@phasecal_sources;
	    push @datasets,$bandpass_source;
	    &source_flux_scale("1934-638",$datadir,$#datasets,@datasets,@if_freq);
	    
	    # and apply the phase calibration to Uranus
	    if ($#phasecal_sources>=0){
		for (my $i=0;$i<=$#phasecal_sources;$i++){
		    &apply_phase_calibration($phasecal_sources[$i],"uranus",$datadir,@if_freq);
		}
	    } else {
		# just scale it to 1934
		&source_flux_scale("1934-638",$datadir,0,"uranus",@if_freq);
	    }
	    
	    if ($backwards_calibration==0){
		# now scale all the datasets by Uranus
		push @datasets,"1934-638";
		push @datasets,"uranus";
		&planet_flux_scale("uranus",$datadir,$#datasets,@datasets,@if_freq);
	    }
	}
	
	# now make a couple of "plots"
	if ($replot_uranus==0){
	    &make_simple_plot("1934-638",$datadir,0,0,@if_freq);
	}
	# check if we should replot
	if ($replot_uranus==1){
	    my @replot_if_freq;
	    for (my $i=0;$i<=$#if_freq;$i++){
		my $chk_file="uranus.".$if_freq[$i].".spectrum";
		if (-e $chk_file){
		    push @replot_if_freq,$if_freq[$i];
		}
	    }
	    @if_freq=@replot_if_freq;
	}

	# we should only plot the shortest baseline for Uranus, so we need to find out
	# which antennae pair that corresponds too
	my ($ant1,$ant2)=&shortest_baseline($ura_response[$observation{'stations'}]);
	&make_simple_plot("uranus",$datadir,$ant1,$ant2,@if_freq);

	# change directory back to what it was
	&change_directory($olddir);

    }
}
$ura->finish;

$dbh->disconnect;

sub extract_all_data {
    my ($dbh,$working_directory,@all_ids)=@_;

    my $idg_query="SELECT * FROM observation WHERE ";
    for (my $i=0;$i<=$#all_ids;$i++){
	if ($i!=0){
	    $idg_query.=" OR ";
	}
	$idg_query.="id=".$all_ids[$i];
    }
    $idg_query.=" ORDER BY time";
    print "QQ executing query: $idg_query\n";
    my $idg=$dbh->prepare($idg_query);
    $idg->execute;
    
    # figure out which headers need to be extracted from each RPFITS file
    my %file_headers;
    my @rpfits_files;
    my $destination_directory;
    my $firsttime;
    my $lasttime;
    my $cycletime;
    my @frequencies;
#    for (my $i=0;$i<=$#all_ids;$i++){
#	my @id_response=&get_id_info($dbh,$all_ids[$i]);
    my $ii=0;
    while (my @id_response=$idg->fetchrow){
	my $rpfits_file=$id_response[$observation{'rpfits_file'}];
	if ($ii==0){
	    @frequencies=&postgresql_array_get($id_response[$observation{'if_freq'}]);
	    $firsttime=$id_response[$observation{'time'}];
	    $destination_directory=$id_response[$observation{'time'}].".".
		$frequencies[0];
	    $destination_directory=~s/\s/\_/g;
	    $cycletime=$id_response[$observation{'cycle_time'}];
	}
	$lasttime=$id_response[$observation{'time'}];
	my $hdrnum=$id_response[$observation{'header_number'}];
	push @{$file_headers{$rpfits_file}},$hdrnum;
	my $already_added=0;
	for (my $j=0;$j<=$#rpfits_files;$j++){
	    if ($rpfits_files[$j] eq $rpfits_file){
		$already_added=1;
	    }
	}
	if ($already_added==0){
	    push @rpfits_files,$rpfits_file;
	}
	$ii++;
    }
    $idg->finish;

    # make the directory if necessary
    my $output_directory=$working_directory."/".$destination_directory;
    my $keep=$output_directory;
    my $keepnumber=1;
    my $directory_made=0;
    if ($replot_uranus==1){
	return $output_directory;
    }
    while ($directory_made==0){
	if (-d $output_directory){
	    # this directory already exists! but we don't want to
	    # use it, as it may already contain another dataset, so
	    # we append and continue
	    $output_directory=$keep.".".$keepnumber;
	    $keepnumber++;
	} else {
	    # we need to make this directory
	    my $mkdir_command="mkdir ".$output_directory;
	    system $mkdir_command;
	    $directory_made=1;
	}
    }

    # change to this directory
    my $olddir=&change_directory($output_directory);
    
    # do the extraction
    my @filesout;
    for (my $i=0;$i<=$#rpfits_files;$i++){
	my $outfile=&strip_path($rpfits_files[$i]).".out";
	my $extract_command="rpfex -o ".$outfile." ";
	my @extract_headers=@{$file_headers{$rpfits_files[$i]}};
	$extract_command.="-q -x";
	for (my $j=0;$j<=$#extract_headers;$j++){
	    if ($j!=0){
		$extract_command.=",";
	    }
	    $extract_command.=$extract_headers[$j];
	}
	$extract_command.=" ".$rpfits_files[$i];
	print "EE executing command: $extract_command\n";
	system $extract_command;
	push @filesout,$outfile;
    }

    # do we have more than one file?
    if ($#filesout>0){
	# sort the files based on date and cat them together in the right
	# time order
	my @sorted_filesout=sort @filesout;
	my $cat_command="cat ";
	for (my $i=0;$i<=$#sorted_filesout;$i++){
	    $cat_command.=$sorted_filesout[$i]." ";
	}
	$cat_command.=" > data.out";
	print "EE executing command: $cat_command\n";
	system $cat_command;
	# remove the individual files
	my $remove_command="rm -f ";
	for (my $i=0;$i<=$#sorted_filesout;$i++){
	    $remove_command.=$sorted_filesout[$i]." ";
	}
	print "EE executing command: $remove_command\n";
	system $remove_command;
    } else {
	# rename the file
	my $rename_command="mv ".$filesout[0].
	    " data.out";
	print "EE executing command: $rename_command\n";
	system $rename_command;
    }

    # time for ATLOD!
    my $atlod_command="atlod in=data.out ".
	"out=data.uv options=birdie,rfiflag,xycorr,opcorr,noauto";
    print "EE executing command: $atlod_command\n";
    system $atlod_command;

    # do we have a 3mm observation?
    if ($frequencies[0]>80000){
	my $atfix_command="atfix vis=data.uv tsyscal=any out=datafix.uv";
	print "EE executing command: $atfix_command\n";
	system $atfix_command;
	my $remove_command="rm -rf data.uv";
	print "EE executing command: $remove_command\n";
	system $remove_command;
	my $rename_command="mv datafix.uv data.uv";
	print "EE executing command: $rename_command\n";
	system $rename_command;
    }

    # flag the dataset based on events
    my @flagevents=&find_nearest_events($dbh,$lasttime,@frequencies);
    # the nearest before is what we're after, we'll flag from the first
    # time to the nearest event before the last time, plus 4 cycles
    my $buffer_time=4*$cycletime; # in seconds
    # we ask postgresql for the time answers since it knows how to do it :)
    my $gtq_query="SELECT TIMESTAMP '".$flagevents[0]."' + INTERVAL '".$buffer_time." second'";
    my $gtq=$dbh->prepare($gtq_query);
    $gtq->execute;
    
    my $final_time;
    while(my @times=$gtq->fetchrow){
	$final_time=$times[0];
    }
    $gtq->finish;
    my $uvflag_command="uvflag ";
    $uvflag_command.=" vis=data.uv";
    $uvflag_command.=" flagval=flag";
    $uvflag_command.=" \"select=time(".
	&postgresql_to_miriad_time($firsttime).",".
	&postgresql_to_miriad_time($final_time).")\"";
    print "EE executing $uvflag_command\n";
    system $uvflag_command;

    # now split the dataset on frequency; it is here we discard antenna 6 data, as we
    # won't be needing it (will overresolve Uranus)
    my $uvsplit_command="uvsplit vis=data.uv \"select=-ant(6)\" options=nosource";
    print "EE executing command: $uvsplit_command\n";
    system $uvsplit_command;

    for (my $i=0;$i<=$#frequencies;$i++){
	# now average the data down to 32 channels
	my $uvaver_command="uvaver vis=uvsplit.".$frequencies[$i]." ".
	    "out=uvsplit.uvaver.".$frequencies[$i]." line=chan,32,1,64";
	print "EE executing: $uvaver_command\n";
	system $uvaver_command;
	# and split it out to source level
	$uvsplit_command="uvsplit vis=uvsplit.uvaver.".$frequencies[$i];
	print "EE executing: $uvsplit_command\n";
	open(UVSPLIT,"-|")||exec $uvsplit_command;
	my @split_sources;
	while(<UVSPLIT>){
	    chomp;
	    my $line=$_;
	    if ($line=~/^Creating/){
		print "MD $line\n";
		my $splitsource=$line;
		$splitsource=~s/^Creating\s(.*)$/$1/;
		push @split_sources,$splitsource;
	    }
	}
	close(UVSPLIT);
	# go through and rename the split sources if they have the wrong
	# frequency appendage
	for (my $j=0;$j<=$#split_sources;$j++){
	    $split_sources[$j]=~/^(.*)\.([^\.]*)$/;
	    my ($this_source,$this_frequency)=($1,$2);
	    if ($this_frequency!=$frequencies[$i]){
		# need to rename this dataset
		my $rename_command="mv ".$split_sources[$j]." ".
		    $this_source.".".$frequencies[$i];
		print "EE executing: $rename_command\n";
		system $rename_command;
	    }
	}
	# and remove the unaveraged split data
	my $remove_command="rm -rf uvsplit.".$frequencies[$i];
	print "EE executing: $remove_command\n";
	system $remove_command;
    }

    # change back to the old directory
    &change_directory($olddir);

    # return the directory where the data is
    return $output_directory;
}

sub change_directory {
    my ($chtodir)=@_;
    my $currentdir=getcwd();
    chdir $chtodir;
    return $currentdir;
}

sub strip_path {
    my ($filename)=@_;
    
    $filename=~s/^.*\/([^\/]*)$/$1/;

    return $filename;
}

sub get_id_info {
    my ($dbh,$get_id)=@_;

    my $get_query="SELECT * FROM observation WHERE id=".$get_id;
#    print "executing query: $get_query\n";
    my $get=$dbh->prepare($get_query);
    $get->execute;
    my @get_response;
    while (@get_response=$get->fetchrow){
	last;
    }
    $get->finish;

    return @get_response;
}

sub array_to_text {
    # turns an array into a postgresql compliant input
    my (@array)=@_;

    my $string="{";
    # check for letters, but we don't look for 'E' which may be used
    # in exponential numbers
    my $needsquotes=0;
    for (my $i=0;$i<=$#array;$i++){
	if ($array[$i]=~/[a-zA-DF-Z]/){
	    $needsquotes=1;
	}
    }
    for (my $i=0;$i<=$#array;$i++){
	if ($i>0){
	    $string.=",";
	}
	if ($needsquotes==1){
	    $string.="'";
	}
	if (!$array[$i]){
	    if ($needsquotes==1){
		$string.="NA";
	    } else {
		$string.="999";
	    }
	} else {
	    $string.=$array[$i];
	}
	if ($needsquotes==1){
	    $string.="'";
	}
    }
    $string.="}";
    return $string;
}

sub postgresql_array_get {
    my ($arraystring)=@_;
    # turns a postgresql array string into a perl array

    $arraystring=~s/[\{\}]//g;
    my @els=split(/\,/,$arraystring);
    return @els;
}

sub time_to_unixtime {
    my ($ptime)=@_;
    
    $ptime=~/^(\d{4})\-(\d{2})\-(\d{2})\s(\d{2})\:(\d{2})\:(\d{2})/;
    my ($year,$month,$day,$hour,$minute,$second)=($1,$2,$3,$4,$5,$6);
    $year-=1900;
    $month-=1;

    my $utime=mktime($second,$minute,$hour,$day,$month,$year,0,0);

    return $utime;
}

sub find_nearest_events {
    my ($dbh,$time,@ifs)=@_;

    my $time_difference="event_time-'".$time."'";
    my ($nearest_before,$nearest_after);
    my $evf_query="SELECT * FROM events WHERE (".
	"((event_type='dcal' OR event_type='acal') AND if_freq='".
	&array_to_text(@ifs)."')) AND EXTRACT(epoch FROM (".$time_difference."))<0".
	" ORDER BY abs(EXTRACT(epoch FROM (".$time_difference."))) LIMIT 1";
#    print "executing query: $evf_query\n";
    my $evf=$dbh->prepare($evf_query);
    $evf->execute;

    while(my @evf_response=$evf->fetchrow){
	$nearest_before=$evf_response[1];
    }
    $evf->finish;

    my $evf_query="SELECT * FROM events WHERE (".
	"((event_type='dcal' OR event_type='acal') AND if_freq='".
	&array_to_text(@ifs)."')) AND EXTRACT(epoch FROM (".$time_difference."))>0".
	" ORDER BY abs(EXTRACT(epoch FROM (".$time_difference."))) LIMIT 1";
#    print "executing query: $evf_query\n";
    my $evf=$dbh->prepare($evf_query);
    $evf->execute;

    while(my @evf_response=$evf->fetchrow){
	$nearest_after=$evf_response[1];
    }
    $evf->finish;

    return ($nearest_before,$nearest_after);
}

sub find_nearest_uranusobs {
    my ($dbh,$time,$nearevent_before,$nearevent_after,@ifs)=@_;

    my $unixtime_reference=&time_to_unixtime($time);
    my @uranus_ids;

    # find an observation with the correct IF configuration nearest to our
    # observation time that also falls between the known nearby event times
    my $endtime="time+totaltime(maxcycles(onsource),maxcycles(offsource),cycle_time)";
    my $fnd_query="SELECT id FROM observation WHERE source='uranus'";
    if ($nearevent_before ne ""){
	$fnd_query.=" AND (".$endtime.")>'".$nearevent_before."'";
    }
    if ($nearevent_after ne ""){
	$fnd_query.=" AND (".$endtime.")<'".$nearevent_after."'";
    }
    $fnd_query.=" AND if_freq='".&array_to_text(@ifs)."' AND el>30".
	" ORDER BY abs(EXTRACT(epoch FROM ((".$endtime.")-'".$time."')))";
#    print "executing query: $fnd_query\n";
    my $fnd=$dbh->prepare($fnd_query);
    $fnd->execute;
    while (my @fnd_response=$fnd->fetchrow){
	push @uranus_ids,$fnd_response[0];
    }
    $fnd->finish;

    return (@uranus_ids);
}

sub find_phasecals {
    my ($dbh,$uranus_id,@ifs)=@_;

    my @phasecal_ids;
    # check the observations before and after the given ID, and see if they
    # are both of a single source at the same frequencies
#    print "trying to get info on phase calibration for ID $uranus_id...\n";
    my @id1_response=&get_id_info($dbh,$uranus_id-1);
    my @id2_response=&get_id_info($dbh,$uranus_id+1);

    if ($id1_response[$observation{'source'}]==
	$id2_response[$observation{'source'}]){
	# well at least the sources match
	# check the frequencies
	my @id1_freqs=&postgresql_array_get($id1_response[$observation{'if_freq'}]);
	my @id2_freqs=&postgresql_array_get($id2_response[$observation{'if_freq'}]);
	my $is_identical=&arrays_are_identical($#id1_freqs,$#id2_freqs,@id1_freqs,
					       @id2_freqs);
	if ($is_identical==0){
	    # the frequencies are the same as each other, but are they the same
	    # as the parent observation
	    my $is_identical1=&arrays_are_identical($#ifs,$#id1_freqs,@ifs,@id1_freqs);
	    if ($is_identical1==0){
		# this is a good candidate for a phase cal, but now we check that
		# the source is a known calibrator
		my $iscal=&source_is_calibrator($dbh,$id1_response[$observation{'source'}]);
		if ($iscal==1){
		    # the source is a known calibrator, so we have a phase calibration!
		    push @phasecal_ids,$uranus_id-1;
		    push @phasecal_ids,$uranus_id+1;
		}
	    }
	}
    }
    
    return @phasecal_ids;
}

sub source_is_calibrator {
    my ($dbh,$sourcename)=@_;

    # check that the source we've been given is a known calibrator
    my $cck_query="SELECT name,type FROM calibrators WHERE name='".$sourcename."'";
#    print "executing query: $cck_query\n";
    my $cck=$dbh->prepare($cck_query);
    $cck->execute;
    my $is_calibrator=0;
    while (my @cck_response=$cck->fetchrow){
	if (($cck_response[0] eq $sourcename)&&
	    ($cck_response[1] eq "p")){
	    $is_calibrator=1;
	}
    }
    $cck->finish;

    return $is_calibrator;
}

sub arrays_are_identical {
    my ($n_array1,$n_array2,@array_stuff)=@_;

    # first check - array sizes are the same?
    if ($n_array1!=$n_array2){
	return 1;
    }

    my @array1;
    my @array2;
    for (my $i=0;$i<=$n_array1;$i++){
	push @array1,shift @array_stuff;
    }
    for (my $i=0;$i<=$n_array2;$i++){
	push @array2,shift @array_stuff;
    }
    
    my $identical=0;
    for (my $i=0;$i<=$n_array1;$i++){
	if ($array1[$i]!=$array2[$i]){
	    $identical=1;
	}
    }

    return $identical;
}

sub find_nearest_fluxcals {
    my ($dbh,$time,$nearevent_before,$nearevent_after,@ifs)=@_;

    my $unixtime_reference=&time_to_unixtime($time);
    my @fluxcal_ids;


    # find an observation with the correct IF configuration nearest to our
    # observation time that also falls between the known nearby event times
    my $endtime="time+totaltime(maxcycles(onsource),maxcycles(offsource),cycle_time)";
    my $fnd_query="SELECT id FROM observation WHERE source='1934-638'";
    if ($nearevent_before ne ""){
	$fnd_query.=" AND (".$endtime.")>'".$nearevent_before."'";
    }
    if ($nearevent_after ne ""){
	$fnd_query.=" AND (".$endtime.")<'".$nearevent_after."'";
    }
    $fnd_query.=" AND if_freq='".&array_to_text(@ifs)."' AND el>30".
	" ORDER BY abs(EXTRACT(epoch FROM ((".$endtime.")-'".$time."')))";
#    print "executing query: $fnd_query\n";
    my $fnd=$dbh->prepare($fnd_query);
    $fnd->execute;
    while (my @fnd_response=$fnd->fetchrow){
	push @fluxcal_ids,$fnd_response[0];
    }
    $fnd->finish;

    return (@fluxcal_ids);
}

sub find_nearest_bandpasscals {
    my ($dbh,$time,$nearevent_before,$nearevent_after,@ifs)=@_;

    my $unixtime_reference=&time_to_unixtime($time);
    my @fluxcal_ids;
    my @fluxcal_sources;

    # which band are we in
    my @highlow_21cm=(0,2000);
    my @highlow_13cm=(2000,4000);
    my @highlow_6cm=(4000,6500);
    my @highlow_3cm=(6500,10000);
    my @highlow_12mm=(15000,25000);
    my @highlow_7mm=(25000,55000);
    my @highlow_3mm=(80000,110000);
    my $bandpass_spec;
    if (($ifs[0]>$highlow_21cm[0])&&($ifs[0]<$highlow_21cm[1])){
	$bandpass_spec="bandpass_21cm";
    } elsif (($ifs[0]>$highlow_13cm[0])&&($ifs[0]<$highlow_13cm[1])){
	$bandpass_spec="bandpass_13cm";
    } elsif (($ifs[0]>$highlow_6cm[0])&&($ifs[0]<$highlow_6cm[1])){
	$bandpass_spec="bandpass_6cm";
    } elsif (($ifs[0]>$highlow_3cm[0])&&($ifs[0]<$highlow_3cm[1])){
	$bandpass_spec="bandpass_3cm";
    } elsif (($ifs[0]>$highlow_12mm[0])&&($ifs[0]<$highlow_12mm[1])){
	$bandpass_spec="bandpass_12mm";
    } elsif (($ifs[0]>$highlow_7mm[0])&&($ifs[0]<$highlow_7mm[1])){
	$bandpass_spec="bandpass_7mm";
    } elsif (($ifs[0]>$highlow_3mm[0])&&($ifs[0]<$highlow_3mm[1])){
	$bandpass_spec="bandpass_3mm";
    }

    # find an observation with the correct IF configuration nearest to our
    # observation time that also falls between the known nearby event times
    my $endtime="observation.time+totaltime(maxcycles(observation.onsource),".
	"maxcycles(observation.offsource),observation.cycle_time)";
    my $fnd_query="SELECT observation.id,observation.source FROM observation,calibrators WHERE ".
	"observation.source=calibrators.name AND calibrators.".$bandpass_spec."=true";
    if ($nearevent_before ne ""){
	$fnd_query.=" AND (".$endtime.")>'".$nearevent_before."'";
    }
    if ($nearevent_after ne ""){
	$fnd_query.=" AND (".$endtime.")<'".$nearevent_after."'";
    }
    $fnd_query.=" AND if_freq='".&array_to_text(@ifs)."' AND el>30 AND observation.source!='0537-441'".
	" ORDER BY abs(EXTRACT(epoch FROM ((".$endtime.")-'".$time."')))";
#    print "executing query: $fnd_query\n";
    my $fnd=$dbh->prepare($fnd_query);
    $fnd->execute;
    while (my @fnd_response=$fnd->fetchrow){
	push @fluxcal_ids,$fnd_response[0];
	push @fluxcal_sources,$fnd_response[1];
    }
    $fnd->finish;

    # which source is the most common?
    my %counts;
    my @sources;
    for (my $i=0;$i<=$#fluxcal_sources;$i++){
	my $already_known=0;
	for (my $j=0;$j<=$#sources;$j++){
	    if ($sources[$j] eq $fluxcal_sources[$i]){
		$already_known=1;
		$counts{$fluxcal_sources[$i]}++;
		last;
	    }
	}
	if ($already_known==0){
	    push @sources,$fluxcal_sources[$i];
	    $counts{$fluxcal_sources[$i]}=1;
	}
    }

    print "DD searching for bandpass calibrators\n";
    my $maxsource=$sources[0];
    my $maxsourcecount=$counts{$sources[0]};
    for (my $i=0;$i<=$#sources;$i++){
	print "DD calibrator ".$sources[$i]." occurs ".$counts{$sources[$i]}." times.\n";
	if ($counts{$sources[$i]}>$maxsourcecount){
	    $maxsourcecount=$counts{$sources[$i]};
	    $maxsource=$sources[$i];
	}
    }
    print "DD selecting source ".$maxsource." as bandpass calibrator\n";

    # remove the other IDs
    for (my $i=0;$i<=$#fluxcal_ids;$i++){
	if ($fluxcal_sources[$i] ne $maxsource){
	    splice @fluxcal_ids,$i,1;
	    splice @fluxcal_sources,$i,1;
	    $i--;
	}
    }

    return (@fluxcal_ids);
}

sub postgresql_to_miriad_time {
    my ($postgresql_time)=@_;
    
    $postgresql_time=~/^([^\-]*)\-([^\-]*)\-([^\-]*)\s([^\:]*)\:([^\:]*)\:(.*)$/;
    my ($year,$month,$day,$hour,$minute,$second)=($1,$2,$3,$4,$5,$6);
    
    my $miriad_time=sprintf "%4d%3s%02d:%02d:%02d:%02d",$year,
    &month_number_to_name($month),$day,$hour,$minute,$second;
    
    return $miriad_time;
}

sub month_number_to_name {
    my ($month_number)=@_;
    
    my @month_names=("JAN","FEB","MAR","APR","MAY","JUN",
		     "JUL","AUG","SEP","OCT","NOV","DEC");
    $month_number--;
    return $month_names[$month_number];
    
}

sub calibrate {
    my ($source,$source_type,$reference_ant,$working_directory,$nifs,@ifs_stuff)=@_;
    my @ifs;
    my @bandifs;
    for (my $i=0;$i<=$nifs;$i++){
	push @bandifs,shift @ifs_stuff;
    }
    @ifs=@ifs_stuff;

    # some MIRIAD commands
    my $miriad_mfcal_command="mfcal";
    my $miriad_gpcal_command="gpcal";

    # preparation
    $working_directory=~s/\/$//;
    my $returncode=0;

    for (my $i=0;$i<=$#ifs;$i++){
	my $calibrated=0;
	my $do_gpcal=1;
	my $no_qusolve=0;
	while($calibrated==0){
	    my $input_dataset=$working_directory."/".$source.".".$ifs[$i];
	    &delete_calibration($input_dataset,$bandifs[$i]);
	    my $mfcal_command=$miriad_mfcal_command;
	    $mfcal_command.=" vis=".$input_dataset;
	    $mfcal_command.=" interval=0.1";
	    $mfcal_command.=" refant=".$reference_ant;
	    if ($bandifs[$i]==0){
		$mfcal_command.=" options=nopassol";
	    }
	    print "EE executing: $mfcal_command\n";
	    open(MFCAL,"-|")|| exec $mfcal_command." 2>&1";
	    my $mfcal_success=1;
	    while(<MFCAL>){
		chomp;
		my $line=$_;
		print "MD $line\n";
		if ($line=~/^\#\#\#\sFatal\sError\:/){
		    $mfcal_success=0;
		}
	    }
	    close(MFCAL);
	    if ($mfcal_success==0){
		$returncode=-1;
		last;
	    }
	    $calibrated=1;
	    if ($do_gpcal==1){
		my $gpcal_command=$miriad_gpcal_command;
		$gpcal_command.=" vis=".$input_dataset;
		$gpcal_command.=" interval=0.1";
		$gpcal_command.=" refant=".$reference_ant;
		$gpcal_command.=" options=xyvary";
		if (($source_type ne "f")&&($no_qusolve==0)){
		    $gpcal_command.=",qusolve";
		}
		print "EE executing: $gpcal_command\n";
		open(GPCAL,"-|")||exec $gpcal_command." 2>&1";
		while(<GPCAL>){
		    chomp;
		    my $line=$_;
		    print "MD $line\n";
		    if ($line=~/\#\#\#\sWarning/){
			if ($line=~/Failed\sto\sconverge/){
			    $calibrated=0;
			    $do_gpcal=0;
			}
		    } elsif ($line=~/^\s+Ant\s.*nan.*/){
			$calibrated=0;
			$do_gpcal=0;
		    } elsif ($line=~/^\#\#\#\sFatal\sError\:/){
			if ($line=~/Solution\sfor\srequested\sparameters\sis\sdegenerate/){
			    $calibrated=0;
			    $no_qusolve=1;
			    $do_gpcal=1;
			} else {
			    $calibrated=0;
			    $do_gpcal=0;
			}
		    }
		}
		close(GPCAL);
	    }
	}
    }

    return $returncode;
}

sub delete_calibration {
    my ($dataset,$delbandpass)=@_;

    # preparation
    $dataset=~s/\/$//;

    my $remove_command="rm -f ".$dataset;
    my @delete_elements=("gains","leakage");
    if ($delbandpass==1){
	push @delete_elements,"bandpass";
    }
    for (my $i=0;$i<=$#delete_elements;$i++){
	my $delete_command=$remove_command."/".$delete_elements[$i];
	print "EE executing: $delete_command\n";
	system $delete_command;
    }
}

sub copy_bandpass_solution {
    my ($bandpass_source,$output_source,$working_directory,@ifs)=@_;

    for (my $i=0;$i<=$#ifs;$i++){
	my $gpcopy_command="gpcopy vis=".$working_directory."/".$bandpass_source.".".
	    $ifs[$i]." out=".$working_directory."/".$output_source.".".$ifs[$i];
	print "EE executing: $gpcopy_command\n";
	system $gpcopy_command;
    }
}

sub apply_phase_calibration {
    my ($phasecal_source,$output_source,$working_directory,@ifs)=@_;

    for (my $i=0;$i<=$#ifs;$i++){
	my $gpcopy_command="gpcopy vis=".$working_directory."/".$phasecal_source.".".
	    $ifs[$i]." out=".$working_directory."/".$output_source.".".$ifs[$i].
	    " mode=merge options=nopass";
	print "EE executing: $gpcopy_command\n";
	system $gpcopy_command;
    }
}

sub planet_flux_scale {
    my ($scale_source,$working_directory,$ndatasets,@stuff)=@_;

    my @datasets;
    for (my $i=0;$i<=$ndatasets;$i++){
	push @datasets,shift @stuff;
    }
    my @ifs=@stuff;

    for (my $i=0;$i<=$#ifs;$i++){
	my $mfboot_command="mfboot \"select=source(".$scale_source.")\" vis=";
	for (my $j=0;$j<=$#datasets;$j++){
	    if ($j!=0){
		$mfboot_command.=",";
	    }
	    $mfboot_command.=$working_directory."/".$datasets[$j].
		".".$ifs[$i];
	}
	$mfboot_command.=" device=/null";
	print "EE executing: $mfboot_command\n";
	system $mfboot_command;
    }

}

sub source_flux_scale {
    my ($scale_source,$working_directory,$ndatasets,@stuff)=@_;

    my @datasets;
    for (my $i=0;$i<=$ndatasets;$i++){
	push @datasets,shift @stuff;
    }
    my @ifs=@stuff;

    for (my $i=0;$i<=$#ifs;$i++){
	for (my $j=0;$j<=$#datasets;$j++){
	    my $gpboot_command="gpboot vis=".$working_directory."/".$datasets[$j].
		".".$ifs[$i]." cal=".$working_directory."/".$scale_source.
		".".$ifs[$i];
	    print "EE executing $gpboot_command\n";
	    system $gpboot_command;
	}
    }
}

sub make_simple_plot {
    my ($source,$working_directory,$ant1,$ant2,@ifs)=@_;

    for (my $i=0;$i<=$#ifs;$i++){
	my $uvspec_command="uvspec vis=".$working_directory."/".$source.".".
	    $ifs[$i]." log=".$working_directory."/".$source.".".$ifs[$i].
	    ".spectrum axis=freq,amp interval=14400 stokes=ii ".
	    "device=/null";
	if (($ant1==0)&&($ant2==0)){
	    $uvspec_command.=" options=nobase,avall";
	} else {
	    $uvspec_command.=" \"select=ant(".$ant1.")(".$ant2.")\"";
	}
	print "EE executing $uvspec_command\n";
	system $uvspec_command;
    }
}

sub max_onsource_time {
    my ($onsourcestring)=@_;

    my @onsourcearray=&postgresql_array_get($onsourcestring);
    my ($minonsource,$maxonsource)=&minmax(@onsourcearray);

    return $maxonsource;
}

sub minmax {
    # get the minimum and maximum values in an array
    my @array=@_;

    my $minval=$array[0];
    my $maxval=$array[0];
    for (my $i=0;$i<=$#array;$i++){
	if ($array[$i]<$minval){
	    $minval=$array[$i];
	}
	if ($array[$i]>$maxval){
	    $maxval=$array[$i];
	}
    }
    return ($minval,$maxval);
}

sub fill_database {
    my ($dbh,$working_directory,@source_ids)=@_;

    for (my $i=0;$i<=$#source_ids;$i++){
	# get info on the ID
	my @id_response=&get_id_info($dbh,$source_ids[$i]);
	# extract the data
	my $extracted_fits=&extract_rpfits($id_response[$observation{'rpfits_file'}],
					   $id_response[$observation{'header_number'}],
					   $id_response[$observation{'source'}],
					   $id_response[$observation{'time'}],$working_directory);
	my @if_freq=&postgresql_array_get($id_response[$observation{'if_freq'}]);
	my @if_bw=&postgresql_array_get($id_response[$observation{'if_bw'}]);
	# change into MIRIAD format
	my @usable_ifs=(0,1);
	my $rc=&fits_to_miriad($extracted_fits,$id_response[$observation{'source'}],
			       $working_directory,@usable_ifs);
	# get some information about the data
	my @datrefs=&get_parameters($id_response[$observation{'source'}],$extracted_fits,
				    $working_directory,@if_freq);
	my %extradata=%{$datrefs[0]};
	# stick the information into the database
	&update_database_metadata($dbh,$source_ids[$i],\%extradata,($#if_freq+1),($#if_freq+1),
				  @if_freq,@if_bw,@if_freq);
	&clean_up($id_response[$observation{'source'}],$extracted_fits,$working_directory,
		  @if_freq);
    }
}

sub clean_up {
    # deletes any files we've created
    my ($source,$rpfits,$working_directory,@ifs)=@_;

    # preparation
    $working_directory=~s/\/$//;
    
    # remove the MIRIAD files and compressed archives
    for (my $i=0;$i<=$#ifs;$i++){
	my $remove_command="rm -rf";
	$remove_command.=" ".$working_directory."/".$source.".".$ifs[$i];
#	if ($output_debug>0){
	    print "executing: $remove_command\n";
#	}
	system $remove_command;
	
#	$remove_command.=".tar.bz2";
#	if ($output_debug>0){
#	    print "executing: $remove_command\n";
#	}
#	system $remove_command;
	
    }

    # remove the RPFITS file
    my $remove_command="rm -rf";
    $remove_command.=" ".$rpfits;
#    if ($output_debug>0){
	print "executing: $remove_command\n";
#    }
    system $remove_command;

}

sub shortest_baseline {
    my ($stations_string)=@_;

    my @stations_array=&postgresql_array_get($stations_string);
    print "DD antennae are at stations: ".&array_to_text(@stations_array)."\n";
    # open the station positions file
    open(STATIONS,"/n/ste616/src/station_coordinates.file");
    my @station_x;
    my @station_y;
    my @station_z;
    my @station_name;
    while(<STATIONS>){
	chomp;
	my @els=split(/\s+/);
	if ($els[0]!~/^\#/){
	    push @station_x,$els[1];
	    push @station_y,$els[2];
	    push @station_z,$els[3];
	    push @station_name,$els[4];
	}
    }
    close(STATIONS);

    my ($short_ant1,$short_ant2,$short_length);
    for (my $i=0;$i<$#stations_array;$i++){
	my ($station_i_x,$station_i_y,$station_i_z);
	for (my $j=0;$j<=$#station_name;$j++){
	    if ($station_name[$j] eq $stations_array[$i]){
		$station_i_x=$station_x[$j];
		$station_i_y=$station_y[$j];
		$station_i_z=$station_z[$j];
		last;
	    }
	}
	for (my $j=$i+1;$j<=$#stations_array;$j++){
	    my ($station_j_x,$station_j_y,$station_j_z);
	    for (my $k=0;$k<=$#station_name;$k++){
		if ($station_name[$k] eq $stations_array[$j]){
		    $station_j_x=$station_x[$k];
		    $station_j_y=$station_y[$k];
		    $station_j_z=$station_z[$k];
		    last;
		}
	    }
	    my $baseline_length=sqrt(($station_j_x-$station_i_x)**2+
				     ($station_j_y-$station_i_y)**2+
				     ($station_j_z-$station_i_z)**2);
	    if ((($i==0)&&($j==1))||
		($baseline_length<$short_length)){
		$short_ant1=$i;
		$short_ant2=$j;
		$short_length=$baseline_length;
	    }
	}
    }

    # antenna numbers start at 1, not 0
    $short_ant1++;
    $short_ant2++;
    print "DD shortest baseline length = $short_length (antennae $short_ant1 - $short_ant2)\n";

    return ($short_ant1,$short_ant2);

}

sub extract_rpfits {
    my ($rpfits_file,$headernum,$source,$datetime,$work_directory)=@_;

    # some commands
    my $rpfits_extract_command="/n/ste616/bin/rpfex";

    # try to find the RPFITS file
    my @search_paths=("/DATA/ARCHIVE_1/ATCA/archive");
    my $found_file="";
    $rpfits_file=~s/^.*\/(.*)$/$1/;

    for (my $i=0;$i<=$#search_paths;$i++){
	$search_paths[$i]=~s/\/$//;
	my $search_file=$search_paths[$i]."/".$rpfits_file;
	if (-e $search_file){
	    $found_file=$search_file;
	    last;
	}
    }
    if ($found_file eq ""){
	# couldn't find this file!
	print "unable to find RPFITS file $rpfits_file from local repository!\n";

	# try to get this file from the ATOA
	$rpfits_file=~/(\d{4})\-(\d{2})\-(\d{2})\_(.*)/;
	my $rpfits_year=$1;
	my $rpfits_month=$2;
	my $rpfits_archive_path="/data/ATOA_1/ATCA";
	if (($rpfits_year>=2009)&&($rpfits_month>7)){
	    $rpfits_archive_path="/data/ATOA_2/ATCA";
	}
	$rpfits_archive_path.="/".$rpfits_year."-".$rpfits_month;

	my $copy_command="rsync -av ste616\@horus:".$rpfits_archive_path."/";
	$copy_command.=$rpfits_file;
	$copy_command.=" ".$work_directory."/";
#	if ($output_debug>0){
	    print "executing: $copy_command\n";
#	}
	system $copy_command;
	
	$found_file=$work_directory."/".$rpfits_file;
	if (!-e $found_file){
	    $found_file="";
	}

    }

    if ($found_file eq ""){
	# still couldn't find the file
	print "unable to find RPFITS file $rpfits_file from any repository!\n";
	
	return "";
    }

    # extract out the data
    $datetime=~s/\s/_/g;
    my $output_file=$source."_".$datetime.".fits";
    $work_directory=~s/\/$//;
    my $output_full=$work_directory."/".$output_file;
    my $extract_command=$rpfits_extract_command;
    $extract_command.=" -x".$headernum.":".$headernum;
    $extract_command.=" -o ".$output_full;
    $extract_command.=" ".$found_file;

#    if ($output_debug>0){
	print "extracting source RPFITS headers with command:\n";
	print $extract_command."\n";
#    }
    if (!-e $output_full){
	system $extract_command;
    } # otherwise the file already exists and has already been extracted

    # return the location of the file we just extracted
    return $output_full;
    
}

sub get_parameters {
    # this routine handles the getting of all the nice metadata we want to stick
    # into the database
    my ($source,$rpfits_file,$working_directory,@ifs)=@_;

    # we get what we can from MIRIAD's uvlist
    my $miriad_uvlist_command="uvlist";
    my $rpfits_rpfhdr_command="rpfhdr";

    # the hash we will return to the caller
    my %parameters;

    # preparation
    $working_directory=~s/\/$//;

    for (my $i=0;$i<=$#ifs;$i++){
	# remove any old uvlist output
	my $logfile="/tmp/uvlist.log";
	my $clear_command="rm -f ".$logfile;
	system $clear_command;
	my $input_dataset=$working_directory."/".$source.".".$ifs[$i];
	my $uvlist_command=$miriad_uvlist_command;
	$uvlist_command.=" vis=".$input_dataset;
	$uvlist_command.=" options=var,full";
	$uvlist_command.=" log=".$logfile;
#	if ($output_debug>0){
	    print "executing: $uvlist_command\n";
#	}
	system $uvlist_command;
	# check that the logfile has been created
	if (!-e $logfile){
#	    if ($output_debug>0){
#		print "unable to get output of uvlist!\n";
#		exit;
	    $parameters{'uvlist_failed'}=1;
#	    }
	} else {
	    $parameters{'uvlist_failed'}=0;
	    open(LOGFILE,$logfile);
	    my $started=0;
	    my $left_to_get=0;
	    my $getting_parameter;
	    my @paramsget;
	    while(<LOGFILE>){
		chomp;
		my $line=$_;
		if ($line=~/^Header\svariables/){
		    $started=1;
		} elsif ($started==1){
		    $line=~s/\:/ /g;
		    $line=~s/(\d)(\-)/$1 $2/g;
		    my @els=split(/\s+/,$line);
		    for (my $j=0;$j<=$#els;$j++){
			if (($els[$j] eq "airtemp")||($els[$j] eq "antaz")||
			    ($els[$j] eq "antel")||($els[$j] eq "chi")||
			    ($els[$j] eq "evector")||($els[$j] eq "jyperk")||
			    ($els[$j] eq "pntdec")||($els[$j] eq "pntra")||
			    ($els[$j] eq "relhumid")||($els[$j] eq "smonrms")||
			    ($els[$j] eq "wind")||($els[$j] eq "winddir")||
			    ($els[$j] eq "nants")){
			    my $this_parameter=$els[$j]."_if".$ifs[$i];
			    $j++;
			    my $this_value=$els[$j];
			    $parameters{$this_parameter}=$this_value;
#			if ($output_debug>1){
			    print "got $this_parameter : $parameters{$this_parameter}\n";
#			}
			} elsif ($left_to_get>0){
			    if ($els[$j] eq $getting_parameter){
				next;
			    }
			    push @paramsget,$els[$j];
			    $left_to_get--;
			    if ($left_to_get==0){
				$getting_parameter.="_if".$ifs[$i];
				$parameters{$getting_parameter}=[@paramsget];
			    }
			} elsif (($els[$j] eq "xtsys")||
				 ($els[$j] eq "xyphase")||($els[$j] eq "ytsys")||
				 ($els[$j] eq "xcaljy")||($els[$j] eq "ycaljy")||
				 ($els[$j] eq "xgtp")||($els[$j] eq "ygtp")||
				 ($els[$j] eq "xsdo")||($els[$j] eq "ysdo")){
			    if ($left_to_get==0){
				$getting_parameter=$els[$j];
				$left_to_get=$parameters{"nants_if$ifs[$i]"};
#			    if ($output_debug>1){
				print "starting parameter get for $getting_parameter, $left_to_get values to get\n";
#			    }
				undef @paramsget;
			    }
			}
		    }
		}
	    }
	    close(LOGFILE);
	}
    }
    
    # we get more parameters from the RPFITS header
    my $rpfhdr_command=$rpfits_rpfhdr_command." ".$rpfits_file;
#    if ($output_debug>0){
	print "executing: $rpfhdr_command\n";
#    }
    my @attenmm_a;
    my @attenmm_b;
    my @subrefl;
    my @pointcor_az;
    my @pointcor_el;
    open(RPFHDR,"-|")||exec $rpfhdr_command;
    while(<RPFHDR>){
	chomp;
	my $line=$_;
	if ($line=~/^ATTEN/){
	    # mm attenuators
	    $line=~/^ATTEN\s+CA0(.)\sMM\=(\d*)\s(\d*)/;
	    my $antenna=$1;
	    my $mmatten_a=$2;
	    my $mmatten_b=$3;
	    $attenmm_a[$antenna-1]=$mmatten_a;
	    $attenmm_b[$antenna-1]=$mmatten_b;
	} elsif ($line=~/^SUBREFL/){
	    # subreflector position
	    $line=~/^SUBREFL\s+CA0(.)\sPOS\=\s*(.*)\sTILT.*/;
	    my $antenna=$1;
	    my $position=$2;
	    $subrefl[$antenna-1]=$position;
	} elsif ($line=~/^POINTCOR/){
	    # pointing correction
	    $line=~/^POINTCOR\s+CA0(.)\sAz\=\s*(.*)\sEl\=\s*(.*)\s.*\/.*/;
	    my $antenna=$1;
	    my $azcor=$2;
	    my $elcor=$3;
	    $pointcor_az[$antenna-1]=$azcor;
	    $pointcor_el[$antenna-1]=$elcor;
	} elsif ($line=~/^CORR_CFG/){
	    # correlator configuration
	    $line=~/^CORR_CFG\=\s*(.*)$/;
	    $parameters{'corr_cfg'}=$1;
	} elsif ($line=~/^POINTINF/){
	    # last pointing information
	    $line=~/^POINTINF\s+(.*)\sSOURCE\=\s*(.*)\s+AZ\=\s(.*)\sEL\=\s(.*)/;
	    $parameters{'point_time'}=$1;
	    $parameters{'point_source'}=$2;
	    $parameters{'point_az'}=$3;
	    $parameters{'point_el'}=$4;
	    $parameters{'point_source'}=~s/\s*$//;
	    print "got pointing time ".$parameters{'point_time'}."\n";
	}
    }
    close(RPFHDR);
    $parameters{'attenmm_a'}=[@attenmm_a];
    $parameters{'attenmm_b'}=[@attenmm_b];
    $parameters{'subrefl'}=[@subrefl];
    # do we need to add the pointing arrays
    if ($parameters{'point_time'}){
	$parameters{'pointcor_az'}=[@pointcor_az];
	$parameters{'pointcor_el'}=[@pointcor_el];
    }
    
    return (\%parameters);
}

sub fits_to_miriad {
    my ($rpfits_chunk,$source,$work_directory,@ifs)=@_;

    # some commands
    my $miriad_atlod_command="atlod";
    my $miriad_uvsplit_command="uvsplit";

    # ATLOD into MIRIAD format
    my @output_files;
    $work_directory=~s/\/$//;
    $rpfits_chunk=~s/^.*\/(.*)$/$1/;
    my $current_directory=getcwd();
    chdir $work_directory;
    my $input_file=$rpfits_chunk;
    my $num_success=0;
    for (my $i=0;$i<=$#ifs;$i++){
	my $this_if=$ifs[$i]+1;
	my $output_file=$input_file."_if".$this_if.".uv";
	my $miriad_options="birdie,rfiflag,noauto,nocacal,xycorr,opcorr";
	my $atlod_command=$miriad_atlod_command;
	$atlod_command.=" in=".$input_file;
	$atlod_command.=" out=".$output_file;
	$atlod_command.=" options=".$miriad_options;
	$atlod_command.=" ifsel=".$this_if;
#	if ($output_debug>0){
	    print "executing command: $atlod_command\n";
#	}
	my $rc=&run_miriad_command($atlod_command);
	if ($rc==0){
	    my $uvsplit_command=$miriad_uvsplit_command;
	    $uvsplit_command.=" vis=".$output_file;
	    $uvsplit_command.=" options=clobber";
#	    if ($output_debug>0){
		print "executing command: $uvsplit_command\n";
#	    }
	    $rc=&run_miriad_command($uvsplit_command);
	    my $remove_command="rm -rf ".$output_file;
#	    if ($output_debug>0){
		print "executing command: $remove_command\n";
#	    }
	    system $remove_command;
	}
	if ($rc==0){
	    $num_success++;
	}
    }

    chdir $current_directory;

    if ($num_success==0){
	# something went wrong
	return -1;
    }
}

sub run_miriad_command {
    my ($miriad_command)=@_;
    
    my $rc=0;
    open(MIRIAD_COMMAND,"-|")||exec $miriad_command." 2>&1";
    while(<MIRIAD_COMMAND>){
	chomp;
	my $line=$_;
#	if ($output_debug>1){
	    print "MIRIAD: $line\n";
#	}
	if ($line=~/^\#\#\#\sFatal\sError/){
	    $rc=-1;
	}
    }
    close(MIRIAD_COMMAND);

    return $rc;
}

sub update_database_metadata {
    my ($dbh,$id,$extradataptr,$n_ifs,$n_usable,@ifs_stuff)=@_;

    my @all_if_freqs;
    my @all_if_bws;
    my @usable_if_freqs;
    for (my $i=0;$i<$n_ifs;$i++){
	push @all_if_freqs,shift @ifs_stuff;
    }
    for (my $i=0;$i<$n_ifs;$i++){
	push @all_if_bws,shift @ifs_stuff;
    }
    for (my $i=0;$i<$n_usable;$i++){
	push @usable_if_freqs,shift @ifs_stuff;
    }

    my %extradata=%{$extradataptr};

    my $sfx="_if".$usable_if_freqs[0];
    # the standard, always present (hopefully!) variables
    my $met_query_variables;
    my $met_query_values;
    if ($extradata{'uvlist_failed'}==0){
	$met_query_variables="az,el,attenmm,subrefl,corr_cfg,airtemp,chi,evector,jyperk,".
	    "pntra,pntdec,relhumid,smonrms,wind,winddir";
	$met_query_values=$extradata{"antaz$sfx"}.",".$extradata{"antel$sfx"}.",'{".
	    &array_to_text(@{$extradata{'attenmm_a'}}).",".
	    &array_to_text(@{$extradata{'attenmm_b'}})."}','".
	    &array_to_text(@{$extradata{'subrefl'}})."','".$extradata{'corr_cfg'}."',".
	    $extradata{"airtemp$sfx"}.",".$extradata{"chi$sfx"}.",".$extradata{"evector$sfx"}.",".
	    $extradata{"jyperk$sfx"}.",".$extradata{"pntra$sfx"}.",".$extradata{"pntdec$sfx"}.",".
	    $extradata{"relhumid$sfx"}.",".$extradata{"smonrms$sfx"}.",".$extradata{"wind$sfx"}.",".
	    $extradata{"winddir$sfx"};
    } else {
	$met_query_variables="attenmm,subrefl,corr_cfg";
	$met_query_values="'{".&array_to_text(@{$extradata{'attenmm_a'}}).",".
	    &array_to_text(@{$extradata{'attenmm_b'}})."}','".
	    &array_to_text(@{$extradata{'subrefl'}})."','".$extradata{'corr_cfg'}."'";
    }
    # variables that may or may not be present
    my @maybe_present=("xtsys","ytsys","xyphase","xcaljy","ycaljy","xgtp","ygtp",
		       "xsdo","ysdo");
    for (my $i=0;$i<=$#maybe_present;$i++){
	if ($extradata{"$maybe_present[$i]$sfx"}){
	    my $tsys_string="'{";
	    for (my $j=0;$j<=$#all_if_freqs;$j++){
		if ($j!=0){
		    $tsys_string.=",";
		}
		if ($all_if_bws[$j]!=2048){
		    $tsys_string.="{";
		    for (my $k=0;$k<$extradata{"nants_if$all_if_freqs[$j]"};$k++){
			if ($k!=0){
			    $tsys_string.=",";
			}
			$tsys_string.="NULL";
		    }
		    $tsys_string.="}";
		} else {
		    my $added=0;
		    for (my $k=0;$k<=$#usable_if_freqs;$k++){
			if ($usable_if_freqs[$k]==$all_if_freqs[$j]){
			    $tsys_string.=&array_to_text(@{$extradata{"$maybe_present[$i]\_if$all_if_freqs[$j]"}});
			    $added=1;
			}
		    }
		    if ($added==0){
			$tsys_string.="{";
			for (my $k=0;$k<$extradata{"nants_if$all_if_freqs[$j]"};$k++){
			    if ($k!=0){
				$tsys_string.=",";
			    }
			    $tsys_string.="NULL";
			}
			$tsys_string.="}";
		    }
		}
	    }
	    $tsys_string.="}'";
	    $met_query_variables.=",$maybe_present[$i]";
	    $met_query_values.=",".$tsys_string;
	}
    }
    if ($extradata{'point_time'}){
	$met_query_variables.=",pointcor,point_time,point_source,point_az,point_el";
	$met_query_values.=",'{".&array_to_text(@{$extradata{'pointcor_az'}}).",".
	    &array_to_text(@{$extradata{'pointcor_el'}})."}','".$extradata{'point_time'}."','".
	    $extradata{'point_source'}."',".$extradata{'point_az'}.",".
	    $extradata{'point_el'};
    }
    my $met_query="UPDATE observation SET (".$met_query_variables.")".
	"= (".$met_query_values.") WHERE id=".$id;
#    if ($output_debug>0){
	print "executing query: $met_query\n";
#    }
    my $met=$dbh->prepare($met_query);
    $met->execute;
    $met->finish;
    
}
